import requests
from lxml import etree


def page_article(num):
    url = "https://news.zhipin.com/wapi/zpgeek/article/querylist.json"
    article_url_format = "https://www.zhipin.com/article/%d.html"
    headers = {
        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
    }
    for i in range(1, num + 1):
        data = {
            "page": i
        }
        response = requests.get(url, params=data, headers=headers)
        resp_json = response.json()
        if resp_json['code'] != 0:
            print(resp_json)
            continue
        for record in resp_json['zpData']['articleList']:
            article_url = article_url_format % (record['id'])
            article_response = requests.get(article_url)
            with open(f"data/{record['id']}.html", mode="w") as fs:
                fs.write(article_response.text)


def parse_article(html):
    tree = etree.HTML(html)
    tree.xpath("//div[@class='detail-content']")
    pass


if __name__ == "__main__":
    # page_article(5)
    with open("data/2306.html",mode="r") as fs:
        content = fs.read()

    tree = etree.HTML(content)
    content_html = tree.xpath("//div[@class='detail-content']/text()")
    print(content_html)

